﻿using IRepository.SystemDefault;
using Model.SystemDefault;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Linq;
using System.Threading.Tasks;
using Common.Net;
using System.Reflection;
using OpenInterface;

namespace Service
{
    public class CrawlerService
    {
        private  IHttpConfigRepository httpConfigRepository = null;
        private  IDataQueueRepository dataQueueRepository = null;
        private  ICollectorUrlRepository collectorUrlRepository = null;
        private  ICollectInfoConfigRepository collectInfoConfigRepository = null;
        private  IPageInfoRepository pageInfoRepository = null;
        private  ICollectorTaskRepository collectorTaskRepository = null;

        public void Init()
        {
            InitRepository();
        }
       
        /// <summary>
        /// 初始化仓库
        /// </summary>
        private  void InitRepository()
        {

            httpConfigRepository = ServiceManager.GetService<IHttpConfigRepository>();
            dataQueueRepository = ServiceManager.GetService<IDataQueueRepository>();
            collectorUrlRepository = ServiceManager.GetService<ICollectorUrlRepository>();
            collectInfoConfigRepository = ServiceManager.GetService<ICollectInfoConfigRepository>();
            pageInfoRepository = ServiceManager.GetService<IPageInfoRepository>();
            collectorTaskRepository = ServiceManager.GetService<ICollectorTaskRepository>();

        }
        

        /// <summary>
        /// 任务启动程序
        /// </summary>
        /// <param name="setupConfigModel"></param>
        public  void SetUp(SetupConfigModel setupConfigModel)
        {

            Console.WriteLine("Setup!");
            while (true)
            {
                if (setupConfigModel == null)
                {
                    Console.WriteLine("SetupConfig 为null");
                    return;
                }

                Thread.Sleep(2000);

                try
                {
                    //取还没有处理的任务
                    //var tasks = collectorTaskRepository.Find(ct => ct.CollectStatus == CollectStatus.Default, constrDataCollectUrRead, false);
                    var tasks = collectorTaskRepository.GetCollecttorTaskListByStatus(CollectStatus.Default, setupConfigModel.ConstrDataRead);
                    if (tasks.Count() > 0)
                    {
                        foreach (var task in tasks)
                        {
                            if (!setupConfigModel.HandlerTasks.Split(',').ToList().Contains(task.CollectorConfigID.ToString())) continue;
                            //var configModel = collectInfoConfigRepository.Find(conf => conf.ID.Equals(task.CollectorConfigID), constrDataCollectUrRead).FirstOrDefault();
                            var configModel = collectInfoConfigRepository.Find(task.CollectorConfigID, setupConfigModel.ConstrDataRead);
                            if (configModel != null)
                            {
                                /*
                                 * 将任务插入队列，并且 更新任务状态
                                 * 
                                 */
                                Console.WriteLine("***new task***:" + configModel.Name);
                                //这个最新更新时间，将来通过web来控制时，在设置任务开启前就应该重置这个时间，避免导致其它任务判断时间错误执行处理
                                configModel.LatestCollectTime = DateTime.Now;
                                collectInfoConfigRepository.Update(configModel, setupConfigModel.ConstrDataWrite);


                                dataQueueRepository.AddOriginalCollectUrlInfo(configModel.ID, new CollectorUrlModel
                                {
                                    AddTime = DateTime.Now,
                                    CollectorConfigID = configModel.ID,
                                    CollectStatus = CollectStatus.Default,
                                    Depth = 0,
                                    ErrorTimes = 0,
                                    Url = configModel.StartUrl

                                }, setupConfigModel.ConstrQueueWrite);
                                //collectorTaskRepository.Remove(task, constrDataCollectUrWrite);

                                task.CollectStatus = CollectStatus.Collected;
                                collectorTaskRepository.Update(task, setupConfigModel.ConstrDataWrite);
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }

            }
        }
        /// <summary>
        /// 地址校验程序
        /// </summary>
        /// <param name="urlCheckerConfigModel"></param>
        public  void UrlChecker(UrlCheckerConfigModel urlCheckerConfigModel)
        {
            Console.WriteLine("UrlChecker");
            var configInfoIndex = 0;
            var configInfoID = 0;

            while (true)
            {
                Thread.Sleep(200);
                if (urlCheckerConfigModel == null)
                {
                    Console.WriteLine("UrlChecker 配置信息无效");
                    return;
                }
                List<int> ConfigIdList = urlCheckerConfigModel.HandlerTasks.Split(',').Select(s => int.Parse(s)).ToList();


                if (ConfigIdList.Count > 0)
                {
                    if (configInfoIndex >= ConfigIdList.Count) configInfoIndex = 0;
                    configInfoID = ConfigIdList[configInfoIndex++];
                }

                try
                {
                    //var configModel = collectInfoConfigRepository.Find(conf => conf.ID.Equals(urlModel.CollectorConfigID), constrDataCollectUrRead).FirstOrDefault();
                    var configModel = collectInfoConfigRepository.Find(configInfoID, urlCheckerConfigModel.ConstrDataRead);

                    //如果 配置设置了停止或者配置无效了，则停止
                    if (configModel == null || configModel.WorkStatus != WorkStatus.Available) continue;

                    var urlModel = dataQueueRepository.GetOriginalUrlInfo<CollectorUrlModel>(configInfoID, urlCheckerConfigModel.ConstrQueueRead);
                    if (urlModel == null) continue;



                    if (urlModel.CollectStatus != CollectStatus.ReCollect)
                    {
                        if (urlModel.AddTime < configModel.LatestCollectTime) continue;

                        //已经采集过了的话(url 存在，且 url 的添加日期在配置的最新采集时间之后)，跳过
                        //var existUrlModel = collectorUrlRepository.Find(url => url.CollectorConfigID.Equals(configModel.ID) && url.Url.Equals(urlModel.Url), constrDataCollectUrRead).OrderByDescending(url => url.ID).FirstOrDefault();
                        var existUrlModel = collectorUrlRepository.GetExistUrlInfo(configModel.ID, urlModel.Url, urlCheckerConfigModel.ConstrDataRead);
                        //这里一定要用 >= 因为时间的精度才到 s 所以会导致已开启就有一条插进来，时间就是相等的
                        if (existUrlModel != null && existUrlModel.AddTime >= configModel.LatestCollectTime) continue;
                    }

                    //深度 +1
                    urlModel.Depth++;
                    //达到指定深度则停止
                    if (configModel.Depth < urlModel.Depth) continue;
                    //default 未 未处理状态
                    urlModel.CollectStatus = CollectStatus.Default;
                    //保存到数据库
                    collectorUrlRepository.Add(urlModel, urlCheckerConfigModel.ConstrDataWrite);
                    //添加到采集队列
                    dataQueueRepository.AddCollectCollectUrlInfo(configInfoID, urlModel, urlCheckerConfigModel.ConstrQueueWrite);
                    Console.WriteLine("@@@@@@AddCollectCollectUrlInfo:" + urlModel.Url);
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }
        }

        #region collector
        /// <summary>
        /// 采集程序
        /// </summary>
        /// <param name="collectorConfigModel"></param>
        public void Collector(CollectorConfigModel collectorConfigModel)
        {
            Console.WriteLine("Collector");

            var tasks = new List<Task>();
            var configInfoIndex = 0;
            var configInfoID = 0;


            while (true)
            {
                Thread.Sleep(200);
                if (collectorConfigModel == null)
                {
                    Console.WriteLine("Collector 配置信息无效");
                    return;
                }
                List<int> ConfigIdList = collectorConfigModel.HandlerTasks.Split(',').Select(s => int.Parse(s)).ToList();
                if (ConfigIdList.Count > 0)
                {
                    if (configInfoIndex >= ConfigIdList.Count) configInfoIndex = 0;
                    configInfoID = ConfigIdList[configInfoIndex++];
                }
                try
                {
                    // var configModel = collectInfoConfigRepository.Find(conf => conf.ID.Equals(urlModel.CollectorConfigID), constrDataCollectUrRead).FirstOrDefault();
                    var configModel = collectInfoConfigRepository.Find(configInfoID, collectorConfigModel.ConstrDataRead);

                    if (configModel == null || configModel.WorkStatus != WorkStatus.Available) continue;

                    var urlModel = dataQueueRepository.GetCollectUrlInfo<CollectorUrlModel>(configInfoID, collectorConfigModel.ConstrQueueRead);
                    if (urlModel == null || urlModel.CollectStatus == CollectStatus.Collected) continue;



                    if (configModel.LatestCollectTime > urlModel.AddTime) continue;
                    //默认休息 避免把人家服务器搞死
                    Thread.Sleep(collectorConfigModel.Frequency < 300 ? 300 : collectorConfigModel.Frequency);

                    Console.WriteLine("#####Collector CollectUrl:" + urlModel.Url);

                    var taskData = new object[] { configModel, urlModel, configInfoID };
                    tasks.Add(Task.Factory.StartNew(state =>
                    {
                        var data = state as object[];
                        CollectInfoConfigModel _configModel = data[0] as CollectInfoConfigModel;
                        CollectorUrlModel _urlModel = data[1] as CollectorUrlModel;
                        CollectHandler(collectorConfigModel, (int)data[2], _configModel, _urlModel);

                    }, taskData));

                    //开多线程取采集,限制一下不要太多
                    if (collectorConfigModel.MutiTaskCount < 0)
                        collectorConfigModel.MutiTaskCount = 0;
                    if (collectorConfigModel.MutiTaskCount >= 10)
                        collectorConfigModel.MutiTaskCount = 10;
                    if (tasks.Count > collectorConfigModel.MutiTaskCount)
                    {
                        tasks = tasks.Where(t => !t.IsCanceled && !t.IsCompleted && !t.IsFaulted).ToList();
                        Task.WaitAny(tasks.ToArray());
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }

        }

        private  void CollectHandler(CollectorConfigModel collectorConfigModel, int configId, CollectInfoConfigModel configModel, CollectorUrlModel urlModel)
        {
            var html = string.Empty;
            try
            {
                //采集
                html = HttpHelper.Get(urlModel.Url, httpConfigRepository.GetUserAgent(), configModel.Encoding, httpConfigRepository.GetProxyIP(collectorConfigModel.ConstrDataRead));
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }

            //出错，可能是异常
            if (!html.IsAvailable() || html.Equals("error"))
            {
                //尝试重复采集，错误超过多少次则不再处理
                if (collectorConfigModel.MaxErrorTimes < 1)
                    collectorConfigModel.MaxErrorTimes = 1;

                if (urlModel.ErrorTimes >= collectorConfigModel.MaxErrorTimes) return;
                urlModel.ErrorTimes++;
                collectorUrlRepository.Update(urlModel, collectorConfigModel.ConstrDataWrite);
                dataQueueRepository.AddCollectCollectUrlInfo(configId, urlModel, collectorConfigModel.ConstrQueueWrite);
                return;
            }
            //采集成功 则保存到待解析队列、数据表
            urlModel.CollectStatus = CollectStatus.Collected;
            collectorUrlRepository.Update(urlModel, collectorConfigModel.ConstrDataWrite);
            var pageInfoModel = new PageInfoModel
            {
                CollectorConfigID = configModel.ID,
                CollectorUrlID = urlModel.ID,
                Url = urlModel.Url,
                Depth = urlModel.Depth,
                AnalyzeStatus = AnalyzeStatus.Default,
                Html = html
            };
            pageInfoRepository.Add(pageInfoModel, collectorConfigModel.ConstrDataWrite);
            dataQueueRepository.AddAnalyzePageInfo(configId, pageInfoModel, collectorConfigModel.ConstrQueueWrite);
        }
        #endregion



        public void Analyzer(AnalyerConfigModel analyerConfigModel)
        {

            Console.WriteLine("Analyzer");
            var configInfoIndex = 0;
            var configInfoID = 0;
            while (true)
            {
                Thread.Sleep(200);
                if (analyerConfigModel == null)
                {
                    Console.WriteLine("Analyzer 配置信息无效");
                    return;
                }
                List<int> ConfigIdList = analyerConfigModel.HandlerTasks.Split(',').Select(s => int.Parse(s)).ToList();
                if (ConfigIdList.Count > 0)
                {
                    if (configInfoIndex >= ConfigIdList.Count) configInfoIndex = 0;
                    configInfoID = ConfigIdList[configInfoIndex++];
                }
                try
                {
                    var configModel = collectInfoConfigRepository.Find(configInfoID, analyerConfigModel.ConstrDataRead);
                    if (configModel == null || configModel.WorkStatus != WorkStatus.Available) continue;

                    var pageModel = dataQueueRepository.GetAnalyzePageInfo<PageInfoModel>(configInfoID, analyerConfigModel.ConstrQueueRead);
                    if (pageModel == null) continue;
                    //var configModel = collectInfoConfigRepository.Find(conf => conf.ID.Equals(pageModel.CollectorConfigID), constrDataCollectUrRead).FirstOrDefault();
                    Assembly assembly = Assembly.Load(new AssemblyName("OpenImplement"));
                    IAnalyzer analyzer = null;

                    //默认解析
                    if (configModel.AnalyzeType == AnalyzeType.Default)
                    {
                        Type type = assembly.GetType("OpenImplement.DefaultAnalyzer");
                        analyzer = (IAnalyzer)Activator.CreateInstance(type);
                    }
                    else
                    {
                        //TODO 根据插件解析,默认这里通过反射 OpenImplement，这里还讲继续改进载入指定地址 dll

                        Type type = assembly.GetType(string.Format("OpenImplement.{0}", configModel.PluginPath));
                        analyzer = (IAnalyzer)Activator.CreateInstance(type);
                    }
                    if (analyzer != null)
                    {
                        Console.WriteLine("-----Analyzer Analyze:" + pageModel.Url);
                        analyzer.Analyze(pageModel.Url, pageModel.Html, (url, handlerType) =>
                        {
                            var urlModel = new CollectorUrlModel
                            {
                                Url = url,
                                AddTime = DateTime.Now,
                                CollectorConfigID = configModel.ID,
                                Depth = pageModel.Depth,//深度将在url检查器 +1 
                                ErrorTimes = 0,
                                CollectStatus = CollectStatus.Default
                            };

                            if (handlerType == HandlerType.ReSetDepth) urlModel.Depth -= 1;
                            if (handlerType == HandlerType.ReCollect)
                            {
                                urlModel.ReCollectTimes += 1;
                                urlModel.Depth -= 1;
                                urlModel.CollectStatus = CollectStatus.ReCollect;
                            }
                            //重新采集超过一定次数将放弃
                            if (analyerConfigModel.MaxReCollectTimes < 1)
                                analyerConfigModel.MaxReCollectTimes = 0;
                            if (urlModel.ReCollectTimes < analyerConfigModel.MaxReCollectTimes)
                                dataQueueRepository.AddOriginalCollectUrlInfo(configInfoID, urlModel, analyerConfigModel.ConstrQueueWrite);

                        });

                        //最后更新为已解析
                        pageModel.AnalyzeStatus = AnalyzeStatus.Analyzed;
                        pageInfoRepository.Update(pageModel, analyerConfigModel.ConstrDataWrite);
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(ex.ToString());
                }
            }

        }
    }
}
